/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright (c) 2008-2011, Salzburg Research. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 * - Redistributions of source code must retain the above copyright notice,
 * this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 * - Neither the name of the KiWi Project nor the names of its contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 * Contributor(s):
 * sschaffe
 *
 * File Info:
 * User: sschaffe
 * Date: 2011/01/19
 * Time: 11:06
 *
 * Project: KiWi2, kiwi-core module
 */

package kiwi.core.services.search;

import kiwi.core.api.config.ConfigurationService;
import kiwi.core.api.content.ContentService;
import kiwi.core.api.search.SolrIndexingService;
import kiwi.core.api.transaction.TransactionData;
import kiwi.core.api.triplestore.ResourceService;
import kiwi.core.api.triplestore.TripleStore;
import kiwi.core.model.rdf.KiWiResource;
import kiwi.core.model.rdf.KiWiTriple;
import kiwi.core.model.rdf.KiWiUriResource;
import kiwi.core.qualifiers.event.transaction.AfterCommit;
import kiwi.core.services.search.solr.program.parser.model.Program;
import kiwi.core.services.search.solr.program.parser.model.Rule;
import kiwi.core.services.search.solr.program.parser.src.ParseException;
import kiwi.core.services.search.solr.program.parser.src.Shortcuts;
import kiwi.core.util.MD5;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
import org.apache.solr.common.SolrInputDocument;
import org.hibernate.proxy.HibernateProxy;
import org.slf4j.Logger;

import javax.enterprise.context.ApplicationScoped;
import javax.enterprise.event.Observes;
import javax.inject.Inject;
import javax.inject.Named;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.io.StringReader;
import java.net.MalformedURLException;
import java.util.*;

/**
 * Add file description here!
 * <p/>
 * User: sschaffe
 */
@Named("kiwi.core.solrIndexingService")
@ApplicationScoped
public class SolrIndexingServiceImpl implements SolrIndexingService {

	private static final List<String> INDEXED_CONTENT_MIMETYPES;
	static {
		INDEXED_CONTENT_MIMETYPES = new ArrayList<String>();
		INDEXED_CONTENT_MIMETYPES.add("text/html");
		INDEXED_CONTENT_MIMETYPES.add("text/xhtml");
		INDEXED_CONTENT_MIMETYPES.add("text/plain");
	}

    @Inject
    private Logger log;

    @Inject
    private ResourceService resourceService;

    @Inject
    private ConfigurationService configurationService;

	@Inject
	private ContentService contentService;

	@Inject
	private TripleStore triplestore;

	private Shortcuts parser;

	private static Program program;

    /** singleton map to store an instance of the SOLR server connection per language, see getSolrServer() */
    private static Map<String,SolrServer> solrServers;


    /**
     * used for keeping track of the number of documents that have been stored recently; based on this count,
     * certain maintenance functions will be carried out when certain thresholds are reached, e.g. optimizing the
     * index.
     */
    private static int docCounter;


    private Set<String> languages;

    public void initialize() throws ParseException {
		//set languages
        languages = new HashSet<String>(configurationService.getListConfiguration("solr.cores"));
        log.info("Starting SOLR Indexing Service for languages {}",languages);
		//set program
		String program_string = (String)configurationService.getConfiguration("solr.conf.program");
		this.parse(new StringReader(program_string));
    }

	/**
	 * set program
	 * @param in program
	 * @throws ParseException
	 */
	public void parse(InputStream in) throws ParseException {
		if(parser != null) {
			parser.ReInit(in);
		} else {
			parser = new Shortcuts(in);
		}
		program = parser.start();
		String p = stringify(program);
		configurationService.setConfiguration("solr.conf.program",p);
		log.info("set shortcut program:\n"+p);
	}

	/**
	 * set program
	 * @param r program
	 * @throws ParseException
	 */
	public void parse(Reader r) throws ParseException {
		if(parser != null) {
			parser.ReInit(r);
		} else {
			parser = new Shortcuts(r);
		}
		program = parser.start();
		String p = stringify(program);
		configurationService.setConfiguration("solr.conf.program",p);
		log.info("set shortcut program:\n"+p);
	}


    /**
     * Apply the updates that are contained in the transaction data to the SOLR search index that is used for
     * full-text search.
     * <p/>
     * The method is implemented as an observer of the @AfterCommit event and thus performs the indexing when
     * changes to resources, triples and namespaces in the master database have been persisted already.
     *
     * @param data
     */
    @Override
    //@Asynchronous
    public void updateIndex(@Observes @AfterCommit TransactionData data) {

        // first, identify resources we need to reindex; we choose all those that are used as subjects in one
        // of the triples contained in either the added or the removed triples list
        Set<KiWiResource> updatedResources = new HashSet<KiWiResource>();

        for(KiWiTriple triple : data.getAddedTriples()) {
            updatedResources.add(triple.getSubject());
        }
        for(KiWiTriple triple : data.getRemovedTriples()) {
            updatedResources.add(triple.getSubject());
        }

        // then, we index each of the resources in turn
        indexResources(updatedResources);
    }

    /**
     * Rebuild the SOLR index from scratch, using the currently visible triples in the triple store.
     */
    @Override
    public void rebuildIndex() {
        // first, clear the SOLR index
        clear();

        // then use the resource service listResources to retrieve resources in batches of 100 and send them to
        // indexing
        int offset = 0;
        Collection<KiWiResource> resources;
        do {
            resources = resourceService.listResources(offset,configurationService.getIntConfiguration("solr.batchsize",100));

            indexResources(resources);

            offset += resources.size();

        } while(resources.size() > 0);

    }

    /**
     * Return an instance of the connection to the SOLR server. Getter method for a singleton, connection
     * established on first call.
     * @return
     */
    private synchronized SolrServer getSolrServer(String language) {
        if(!languages.contains(language)) {
            log.warn("language {} not supported; reverting to generic SOLR index",language);
            language = "generic";
        }

        if(solrServers == null) {
            solrServers = new HashMap<String,SolrServer>();
        }
        SolrServer solrServer = solrServers.get(language);

        if(solrServer == null) {
            String core = configurationService.getStringConfiguration("solr.core."+language);
            try {
                solrServer = new CommonsHttpSolrServer(configurationService.getServerUri()+"solr/"+core);
            } catch (MalformedURLException e) {
                log.error("error while establishing connection to SOLR webservice (language {}); SOLR search not available", language);
            }
        }
        return solrServer;
    }


    /**
     * Store the resources passed as argument in the SOLR index.
     */
    private void indexResources(Collection<KiWiResource> resources) {
        List<String> languages = configurationService.getListConfiguration("solr.cores");

        // for each language a set of input documents that we use for batch processing
        Map<String,Set<SolrInputDocument>> docs = new HashMap<String, Set<SolrInputDocument>>();
        for(String language : languages) {
            docs.put(language, new HashSet<SolrInputDocument>());
        }

        int docSize = 0;

        // used for persisting in batches of 100 ...
        int batchSize  = configurationService.getIntConfiguration("solr.batchsize",100);

        for(KiWiResource resource : resources) {
             // create the SOLR representation of the resource

            Map<String,SolrInputDocument> resourceDocuments = createDocument(resource,languages);

            for(String language : languages) {
                docs.get(language).add(resourceDocuments.get(language));
            }

            docSize ++;

            // if the batchsize is reached, send the documents to the server and clear the list
            if(docSize >= batchSize) {
                for(String language : languages) {
                    SolrServer server               = getSolrServer(language);
                    Set<SolrInputDocument> langDocs = docs.get(language);
                    try {
                        synchronized (server) {
                            server.add(langDocs);
                            server.commit();
                        }

                        langDocs.clear();

                    } catch (SolrServerException e) {
                        log.error("document could not be indexed: a SOLR Exception occurred (server not available?)",e);
                    } catch (IOException e) {
                        log.error("document could not be indexed: an I/O Exception occurred",e);
                    } catch (Exception e) {
                        log.error("document could not be indexed: a runtime Exception occurred (server sending invalid response?)",e);
                    }
                }
                docSize = 0;
            }
        }

        // if the batchsize is reached, send the documents to the server and clear the list
        if(docSize > 0) {
            for(String language : languages) {
                SolrServer server               = getSolrServer(language);
                Set<SolrInputDocument> langDocs = docs.get(language);
                try {
                    synchronized (server) {
                        server.add(langDocs);
                        server.commit();
                    }

                    langDocs.clear();

                } catch (SolrServerException e) {
                    log.error("document could not be indexed: a SOLR Exception occurred (server not available?)",e);
                } catch (IOException e) {
                    log.error("document could not be indexed: an I/O Exception occurred",e);
                } catch (Exception e) {
                    log.error("document could not be indexed: a runtime Exception occurred (server sending invalid response?)",e);
                }
            }
            docSize = 0;
        }

        docCounter += resources.size();

        optimize();

    }


    private void clear() {
        for(String language : configurationService.getListConfiguration("solr.cores")) {
            // server connection for indexing
            SolrServer server = getSolrServer(language);

            try {
                synchronized (server) {
                    server.deleteByQuery("id:[* TO *]");
                }
            } catch (IOException e) {
                log.error("could not optimize search index: an I/O Exception occurred",e);
            } catch (SolrServerException e) {
                log.error("could not optimize search index: a SOLR Exception occurred (server not available?)",e);
            } catch (Exception e) {
                log.error("document could not be indexed: a runtime Exception occurred (server sending invalid response?)",e);
            }
        }

    }

    /**
     * Optimize the SOLR index if the threshold for optimization ("solr.optimize" configuration) is reached.
     */
    private void optimize() {
        if(docCounter >= configurationService.getIntConfiguration("solr.optimize",1000)) {
            for(String language : configurationService.getListConfiguration("solr.cores")) {

                // server connection for indexing
                SolrServer server = getSolrServer(language);

                try {
                    server.optimize();
                } catch (IOException e) {
                    log.error("could not optimize search index: an I/O Exception occurred",e);
                } catch (SolrServerException e) {
                    log.error("could not optimize search index: a SOLR Exception occurred (server not available?)",e);
                } catch (Exception e) {
                    log.error("document could not be indexed: a runtime Exception occurred (server sending invalid response?)",e);
                }
            }
        }
    }


	private Map<String,SolrInputDocument> createDocument(KiWiResource resource, Collection<String> languages) {
		Map<String,SolrInputDocument> result = new HashMap<String,SolrInputDocument>();
		//for(String language : languages) {
			String language = "generic";
			SolrInputDocument doc = new SolrInputDocument();
        	result.put(language,doc);

			doc.addField("id",resource.getId());
			doc.addField("created",resource.getCreated());
			doc.addField("author_name",resource.getCreator().getFirstName()+" "+resource.getCreator().getLastName());
			doc.addField("author_login",resource.getCreator().getLogin());
			//doc.addField("modified",resource.);???

			if (resource.isUriResource()) {
                KiWiUriResource r;
                // workaround for hibernate lazy proxies that are not instances of
                // KiWiUriResource
                if (resource instanceof HibernateProxy) {
                    r = (KiWiUriResource) ((HibernateProxy) resource)
                            .getHibernateLazyInitializer().getImplementation();
                } else {
                    r = (KiWiUriResource) resource;
                }

				//add content if there is one
				//TODO maybe content must be preprocessed for indexing
				String mimetype = contentService.getContentType(r);
				if(r!=null && INDEXED_CONTENT_MIMETYPES.contains(mimetype)) {
					byte[] c = contentService.getContent(r,mimetype);
					String content = new String(c,0,c.length);
					doc.addField("content",content);
				}
                doc.addField("uri", r.getUri());
            }

			//set shortcut fields
			for(Rule rule : program) {
				rule.apply(triplestore,resource,resourceService,doc);
			}
		//}
		return result;
	}
	/*
    private Map<String,SolrInputDocument> createDocument(KiWiResource resource, Collection<String> languages) {

        Map<String,SolrInputDocument> result = new HashMap<String,SolrInputDocument>();

        for(String language : languages) {
            SolrInputDocument doc = new SolrInputDocument();
            result.put(language,doc);

            doc.addField("id", resource.getId());

            Locale locale = null;
            if(language != null && !language.equals("generic")) {
                locale = new Locale(language);
            }

            if (resource.isUriResource()) {
                KiWiUriResource r;
                // workaround for hibernate lazy proxies that are not instances of
                // KiWiUriResource
                if (resource instanceof HibernateProxy) {
                    r = (KiWiUriResource) ((HibernateProxy) resource)
                            .getHibernateLazyInitializer().getImplementation();
                } else {
                    r = (KiWiUriResource) resource;
                }

                doc.addField("uri", r.getUri());
            }

            // some commonly used specialised fields
            String title = resourceService.getLabel(resource,locale);
            if(title != null) {
                doc.addField("title", title, 10);
                doc.addField("title_id", title, 10);
            }

            if (resource.getCreator() != null) {
                doc.addField("author_name", resource.getCreator().getFirstName() + " " + resource.getCreator().getLastName());
                doc.addField("author_login", resource.getCreator().getLogin());
            }
        }


		// RDF properties: for each literal property, the URI is used as key,
		// converted to MD5 sum;
		// we store for each property the following fields:
		// - l_KEY: the literal content of the property as tokenized text
		// - i_key: the integer value of the property (if parsable), sortable
		// and queriable
		// - d_key: the double value of the property (if parsable), sorable and
		// queriable
		// for each URI property, the URI is used as key, converted to MD5 sum;
		// we store for each
		// property the following fields:
		// - u_key: the URI of the Uri resource
        List<KiWiTriple>  outgoing      =  resourceService.listOutgoing(resource);
        Set<String>       kspaces       =  new HashSet<String>();
        Set<KiWiResource> tagCandidates = new HashSet<KiWiResource>();

		for (final KiWiTriple t : outgoing) {

			if (t.getObject().isLiteral()) {
				final KiWiLiteral l = (KiWiLiteral) t.getObject();

                final String key = generatePropertyKey(t.getProperty().getUri());

                for(String language : languages) {
                    SolrInputDocument doc = result.get(language);

                    // the literal is language dependent text, so we only add it if the language matches
                    if(language.equals("generic") || l.getLanguage() == null || l.getLanguage().getLanguage().equals(language) ) {
                        doc.addField("l_" + key,l.getContent());

                        if(l instanceof KiWiTextContentLiteral) {
                            doc.addField("t_" + key,l.getContent());
                            continue;
                        }
                     }

                    if (l instanceof KiWiDoubleLiteral) {
                        doc.addField("d_" + key, ((KiWiDoubleLiteral) l).getDoubleContent());
                    } else if (l instanceof KiWiIntLiteral) {
                        doc.addField("i_" + key, ((KiWiIntLiteral) l).getIntContent());
                    }
                }
			} else if (t.getObject() instanceof KiWiUriResource) {

				final KiWiUriResource u = (KiWiUriResource) t.getObject();

				final String key = generatePropertyKey(t.getProperty().getUri());
                for(String language : languages) {
                    SolrInputDocument doc = result.get(language);
				    doc.addField("u_" + key, u.getUri());
                }
			}

            if (t.getContext() != null) {
                String contextTitle = t.getContext().getTitle();
                kspaces.add(contextTitle);
            }


            // first look for all directly associated tags
            if(t.getProperty().getUri().equals(Constants.NS_HGTAGS+"taggedWithTag")) {
                KiWiNode node = t.getObject();
                if(node.isUriResource() || node.isAnonymousResource()) {
                    tagCandidates.add((KiWiResource)node);
                }

            }
            // TODO: cover the tagging here as well instead of doing it separately below!
		}


        for(String contextTitle : kspaces) {
            for(String language : languages) {
                SolrInputDocument doc = result.get(language);
                doc.addField("kspace",contextTitle);
            }
        }







        // list all tags of the resource; this is a bit complicated since we need to take into account all taggings
        // in absence of the reasoner ...

        try {
            // we added all directly associated tags already in the previous run

            // then list all taggings and look there (needed as long as reasoner is not working properly)
            for(KiWiResource tagging : resourceService.listIncomingNodes(resource, Constants.NS_HGTAGS+"taggedResource")) {
                for(KiWiNode tag : resourceService.listOutgoingNodes(tagging,Constants.NS_HGTAGS+"associatedTag")) {
                    if(tag.isUriResource() || tag.isAnonymousResource()) {
                        tagCandidates.add((KiWiResource)tag);
                    }
                }
            }

            for(KiWiResource tag : tagCandidates) {
                for(String language : languages) {
                    Locale locale = null;
                    if(language != null && !language.equals("generic")) {
                        locale = new Locale(language);
                    }
                    SolrInputDocument doc = result.get(language);

                    for(String label : resourceService.getProperties(tag,Constants.NS_HGTAGS+"name",locale)) {
                        doc.addField("tag",label);
                    }
                    for(String label : resourceService.getProperties(tag,Constants.NS_HGTAGS+"name")) {
                        doc.addField("tag",label);
                    }
                }
            }

        } catch(NamespaceResolvingException ex) {
            log.error("an error occurred while resolving the SKOS namespace; this should not happen and is a programming error.");
        }


        return result;
    }
    */


    /**
     * Generation strategy for generating the keys to use in field names of properties. The reason why we need this
     * is that certain characters that appear in uris are not allowed in field names, e.g. ":". The current i
     * mplementation creates the MD5 sum of the URI and uses it. Disadvantage is that this is not easy to use by
     * the user, so a more reasonable field name is preferrable
     * @param uri
     * @return
     */
    private static String generatePropertyKey(String uri) {
        return MD5.md5sum(uri);
    }

	private static String stringify(List<Rule> program) {
		StringBuffer b = new StringBuffer();
		for(Rule rule : program) {
			b.append(rule.toString());
			b.append("\n");
		}
		return b.toString();
	}

	public static Program Program() {
		return program;
	}

	public static String ProgramString() {
		return stringify(program);
	}

}
